from urllib.parse import urlencode

import requests
from requests import Session, ReadTimeout
from pyquery import PyQuery as pq
from requests.exceptions import ProxyError, ConnectTimeout

from weixin.config import PROXY_POOL_URL, VALID_STATUSES, MAX_FAILED_TIME
from weixin.db import RedisQueue
from weixin.mysql import Mysql
from weixin.request import WeixinRequest

class Spider():
    base_url = 'http://weixin.sogou.com/weixin'
    keyword = '梅西'
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Cache-Control': 'max-age=0',
        'Connection': 'keep-alive',
        'Cookie': 'CXID=5C083E75B18FBC9F3BB8C82D91F489B2; SUID=32CF51DF4D238B0A5CDAB772000062AC; SUV=00B68D80DF51CF2D5CDE5F75DF2EE448; wuid=AAFwNty4JwAAAAqLE2PfVAEAGwY=; ld=LZllllllll2N5msxlllllV1MfD9llllltl64Xlllll9lllll4ylll5@@@@@@@@@@; LSTMV=307%2C150; LCLKINT=3622; ABTEST=0|1565620442|v1; weixinIndexVisited=1; ad=9Zllllllll2tLrNhlllllVCYHqwlllllTC45Gkllll9lllll9Oxlw@@@@@@@@@@@; JSESSIONID=aaaiQXABnxQvtf5Xq9oXw; PHPSESSID=on9o0h0o5693dgvlcnud69at93; sct=3; IPLOC=CN3711; SNUID=7EB8C9CFBFC52FCAAA707764C09DA46F; seccodeRight=success; successCount=2|Mon, 19 Aug 2019 10:05:11 GMT; ppinf=5|1566208923|1567418523|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTozNjolRTYlQUMlQjIlRTglQkUlQTglRTUlQkYlOTglRTclODIlOEV8Y3J0OjEwOjE1NjYyMDg5MjN8cmVmbmljazozNjolRTYlQUMlQjIlRTglQkUlQTglRTUlQkYlOTglRTclODIlOEV8dXNlcmlkOjQ0Om85dDJsdUpqbUVLdWVQT1Y3LTZaTnhza0E0ekFAd2VpeGluLnNvaHUuY29tfA; pprdig=tP_1tGhkoBihhTP3AjGE5O8DxngibBLSGiXNa4XZ_70OXnCFihjM169_wzKbBcY7Mk4BtXd6-yr9z2Q0BmJj0tnj4UZNREqHqIls6MMKaOU09b3fyWR2Bvr0ddC1SJEZbIyTmUwnEn1MTeA0GKJPYgh71TNn3IDjpryfwWUAB7M; sgid=05-40582789-AV1ac5tE8umwNqgpibJYjiaD8; ppmdig=15662089230000006fc0d3d41a3acea37b91ee340b2a9499',
        'Host': 'weixin.sogou.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36',
    }

    redis = RedisQueue()
    session = Session()
    mysql = Mysql()

    def get_proxy(self):
        """
        获取代理
        :return:
        """
        response = requests.get(PROXY_POOL_URL)
        try:
            if response.status_code in VALID_STATUSES:
                print('Get Proxy', response.text)
                return response.text
            return None
        except Exception as e:
            print(e)
            return None

    def request(self,weixin_request):
        """
        利用请求访问
        :param weixin_request:
        :return:
        """
        try:
            if weixin_request.need_proxy:
                proxy = self.get_proxy()
                if proxy:
                    proxies = {
                        'http': 'http://' + proxy,
                        'https': 'https://' + proxy,
                    }
                    return self.session.send(weixin_request.prepare(),
                                             timeout=weixin_request.timeout, allow_redirects=False, proxies=proxies)
            return self.session.send(weixin_request.prepare(),
                                     timeout=weixin_request.timeout, allow_redirects=False)
        except (ConnectionError, ReadTimeout, ConnectTimeout, ProxyError) as e:
            print(e.args)
            return False

    def start(self):
        """
        第一个开始请求
        :return:
        """
        # 全局更新Headers
        self.session.headers.update(self.headers)
        #start_url = self.base_url + '?' + urlencode({'query': self.keyword, 'type': 2})
        start_url = 'http://www.baidu.com'
        weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, need_proxy=True)
        # 调度第一个请求
        self.redis.add(weixin_request)

    def parse_index(self,response):
        """
        解析搜索页
        :param response:
        :return:
        """
        html = response.text
        doc = pq(html)

        a_list = doc('#main > div.news-box > ul > li > div.txt-box > h3 > a')
        for item in a_list.items():
            href = item.attr('href')
            print('item: ' + href)
            ####方法可以作为参数传递####
            weixin_request = WeixinRequest(url=href, callback=self.parse_detail)
            self.redis.add(weixin_request)
            yield weixin_request

        next_page = doc('#sogou_next')
        url = next_page.attr('href')
        ####方法可以作为参数传递####
        weixin_request = WeixinRequest(url=href, callback=self.parse_index, need_proxy=True)
        self.redis.add(weixin_request)
        yield weixin_request

    def parse_detail(self, response):
        """
        解析微信公众号内容
        :param response:
        :return:
        """
        html = response.text
        doc = pq(html)
        title = doc('#activity-name').text()
        author = doc('#profileBt').text()
        publish_time = doc('#publish_time').text()
        content = doc('#js_content').text()
        data = {
            'title': title,
            'author': author,
            'publish_time': publish_time,
            'content': content,
        }
        return data

    def error(self, weixin_request):
        """
        错误处理
        :param weixin_request: 请求
        :return:
        """
        weixin_request.fail_time = weixin_request.fail_time + 1
        print('Request Failed', weixin_request.fail_time, 'Times', weixin_request.url)
        if weixin_request.fail_time < MAX_FAILED_TIME:
            print('count:', self.redis.count())
            self.redis.add(weixin_request)
            ###第一次获取出错##
            if self.redis.count() == 1:
                self.schedule()

    def schedule(self):
        """
        调度
        :return:
        """
        if not self.redis.empty():
            weixin_request = self.redis.pop()
            if isinstance(weixin_request, WeixinRequest):
                response = self.request(weixin_request)
                if response and response.status_code in VALID_STATUSES:
                    results = list(weixin_request.callback(response))
                    if results:
                        for result in results:
                            print('New Result', type(result))
                            if isinstance(result, WeixinRequest):
                                self.redis.add(WeixinRequest)
                            if isinstance(result, dict):
                                self.mysql.insert('article', result)
                    self.error(weixin_request)
                else:
                    self.error(weixin_request)

    def run(self):
        """
        入口
        :return:
        """
        self.start()
        self.schedule()

if __name__ == '__main__':
    spider = Spider()
    spider.run()
